package irie.search;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.ansj.splitWord.analysis.ToAnalysis;
import org.apache.log4j.BasicConfigurator;

public class HTMLAnalyser {
	static List<HTMLentity> list = new ArrayList<HTMLentity>();
	static Map<String, Integer> map = new HashMap<String, Integer>();
	static int wordid = 1;
	static int vectorcount;
	static String reg = "[^\u4e00-\u9fa5]";
	static int htmlnum;
	// double[][] W = new double[htmlnum][vectorcount];

	public void input() {
		//如需使用，请修改html.txt文件的路径！
		File file = new File("D:\\webmagic\\entity", "html.txt");
		BufferedReader reader = null;
		int linenum = 1;
		try {
			reader = new BufferedReader(new FileReader(file));
			String tempString = null;
			// 逐行读入，一行一个String
			while ((tempString = reader.readLine()) != null) {
				// System.out.println(tempString);
				HTMLentity hen = new HTMLentity();
				// Pattern p = Pattern.compile("http");
				Pattern p = Pattern.compile(
						"(http|ftp|https):\\/\\/[\\w\\-_]+(\\.[\\w\\-_]+)+([\\w\\-\\.,@?^=%&amp;:/~\\+#]*[\\w\\-\\@?^=%&amp;/~\\+#])?");
				Matcher m = p.matcher(tempString);
				m.find();
				// System.out.println(m.group());
				hen.setHid(linenum++);
				hen.setURL(m.group());
				hen.setContent(tempString.replaceAll(reg, ""));
				hen.setFeature(null);
				list.add(hen);
			}
			htmlnum=list.size();
			reader.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public double[][] analyse() {
		this.input();
		Iterator it = list.iterator();
		// 读取一个list元素，对content分词，抽取名词，放入map中
		while (it.hasNext()) {
			HTMLentity he = (HTMLentity) it.next();
			String s = he.getContent();
			s = ToAnalysis.parse(s).toString();
			String s1[] = s.split(",");
			for (int i = 0; i < s1.length; i++) {
				if (s1[i].contains("/n")) {
					s1[i] = s1[i].replaceAll(reg, "");
					if (!map.containsKey(s1[i]))
						map.put(s1[i], wordid++);
				}
			}
		}
		// 对于每个list元素计算出对应于map中元素的向量（计算权重），存入list中
		vectorcount = map.size();
		it = list.iterator();
		int[] wordapr = new int[vectorcount];
		int[][] freq = new int[htmlnum][vectorcount];// i是html文件计数，j是词计数
		for (int i = 0; i < wordapr.length; i++)
			wordapr[i] = 0;
		for (int i = 0; i < htmlnum; i++)
			for (int j = 0; j < vectorcount; j++)
				freq[i][j] = 0;
		// 每个循环处理一个html
		while (it.hasNext()) {
			int[] flag = new int[vectorcount];
			for (int i = 0; i < flag.length; i++) {
				flag[i] = 0;
			}
			HTMLentity he = (HTMLentity) it.next();
			int htmlid = he.getHid() - 1;
			String s = he.getContent();
			s = ToAnalysis.parse(s).toString();
			String s1[] = s.split(",");
			// 每个循环处理一个词
			for (int i = 0; i < s1.length; i++) {
				if (s1[i].contains("/n")) {
					// 查找map中的序号，计算频数，写入list的feature中
					s1[i] = s1[i].replaceAll(reg, "");
					int wloc = map.get(s1[i]) - 1;// 找到这个词在词map中的位置
					if (flag[wloc] == 0) {
						wordapr[wloc]++;
						flag[wloc] = 1;
					}
					freq[htmlid][wloc]++;
				} else
					continue;
			}
		}
		// 计算相关数值
		double[][] F = new double[htmlnum][vectorcount];
		double[] IDF = new double[vectorcount];
		double[][] W = new double[htmlnum][vectorcount];
		for (int i = 0; i < htmlnum; i++)
			for (int j = 0; j < vectorcount; j++) {
				if (freq[i][j] == 0)
					F[i][j] = 0;
				else
					F[i][j] = freq[i][j] / (double) this.calcmax(freq[i]);
			}
		for (int i = 0; i < vectorcount; i++) {
			IDF[i] = Math.log(htmlnum / (double) wordapr[i]);
		}
		// 计算权重
		for (int i = 0; i < htmlnum; i++)
			for (int j = 0; j < vectorcount; j++) {
				W[i][j] = F[i][j] * IDF[j];
			}
		// Test
		// for (int i = 0; i < htmlnum; i++)
		// for (int j = 0; j < vectorcount; j++) {
		// System.out.println(W[i][j]);
		// }
		// for (int i = 0; i < vectorcount; i++) {
		// System.out.println(IDF[i]);
		// }
		// System.out.println(W[99][4000]);
		// 最后yi个html的频率出了问题
		/*
		 * int c = 0; for (int j = 0; j < 100; j++) { c=0; for (int i = 0; i <
		 * vectorcount; i++) { if (freq[j][i] == 0) c++; } if(c==4157)
		 * System.out.println(j+1); }
		 */
		return W;
	}

	public int calcmax(int[] arr) {
		int max = 0;
		for (int i = 0; i < arr.length; i++) {
			if (arr[i] > max)
				max = arr[i];
		}
		return max;
	}
 
	public static void main(String[] args) {
		BasicConfigurator.configure();
		HTMLAnalyser ha = new HTMLAnalyser();
		//ha.input();
		/*
		 * Iterator<HTMLentity> it = list.iterator(); while(it.hasNext()){
		 * HTMLentity he = (HTMLentity) it.next();
		 * System.out.print(he.getHid()); System.out.print(he.getURL());
		 * System.out.println(he.getContent()); }
		 */
		ha.analyse();

		// for (int i = 0; i < htmlnum; i++) for (int j = 0; j < vectorcount;
		// j++) { System.out.println(ha.analyse()[i][j]); }

		// test max function
	}
}
